# coding=utf-8

import threading
from datetime import datetime
from queue import Queue
from time import sleep

from difflib import SequenceMatcher


def p(str1):
    if 0:
        print("%s" % str1)


def _wordcount(dict_, str1):

    found = False
    for key in dict_.keys():
        ret = SequenceMatcher(None, key, str1).quick_ratio()
        if ret >= 0.8 :
            found = True
            dict_[key] += 1
            return
    if not found:
        dict_[str1] = 1


flag1 = 0  # 目标是分段数量
flag2 = 0  # 目标是1
q_dict = Queue()


def do_str_match(lines, start, end):

    global flag1, q_dict
    
    dict_ = {}
    
    p("info: do_str_match(%s, %s) BEGIN" % (start, end))
    
    for i in range(start, end + 1):
#         p("info: do_str_match: %s" % i)
        _wordcount(dict_, lines[i])

    p("info: do_str_match(%s, %s) DONE" % (start, end))
    
    q_dict.put(dict_)
    flag1 += 1


def do_combine_dict(dict1, dict2):
    
    global flag2, q_dict
    
    # 经过多种推断，确认这种更耗时。比如100&10、100&100
#     for key2 in dict2.keys():
#         found = False
#         for key1 in dict1.keys():
#             ret = SequenceMatcher(None, key1, key2).quick_ratio()
#             if ret > 0.8 :
#                 dict1[key1] += dict2[key2]
#                 found = True
#                 break
#             
#         if not found:
#             dict1[key2] = dict2[key2]
            
    for key1 in list(dict1.keys()):
        for key2 in dict2.keys():
            ret = SequenceMatcher(None, key1, key2).quick_ratio()
            if ret > 0.8 :
                dict1[key1] += dict2[key2]
                del dict2[key2]
                break
             
    for key, value in dict2.items():
        dict1[key] = value
    
    q_dict.put(dict1)
    flag2 -= 1
    
    
if __name__ == '__main__':

    # 1.读文件
    with open('1.copy.txt', 'rb') as f:
        lines = f.readlines()
    
    # 2.分段
    nrows = len(lines)
    count = 30
    part = nrows // count
    flag2 = count
    
    # 3.分段统计
    threads = []
    for i in range(0, count):
        begin = i * part
        if i != count - 1:
            end = (i + 1) * part - 1
        else:
            end = nrows - 1
        
        t = threading.Thread(target=do_str_match, args=(lines, begin, end))
        threads.append(t)

    print(datetime.now())

    for t in threads:
        t.start()
        
    # 4.周期轮巡
    while True:
        sleep(3)
        
        # 退出条件：分段统计完成 & 字典合并完成
        print("info: flag1=%s flag2=%s qsize=%s" % (flag1, flag2, q_dict.qsize()))
        if flag1 == count and flag2 == 1:
            break
        
        # 字典合并
        while q_dict.qsize() > 1:
            dict1 = q_dict.get()
            dict2 = q_dict.get()
            if len(dict1) >= len(dict2):
                t = threading.Thread(target=do_combine_dict, args=(dict1, dict2))
            else:
                t = threading.Thread(target=do_combine_dict, args=(dict2, dict1))
            threads.append(t)
            t.start()
          
    for t in threads:
        t.join()

    print(datetime.now())
    
    # 5.排序
    result = sorted(q_dict.get().items(), key=lambda x:x[1], reverse=True)
    
    print(datetime.now())
    
#     print(result)
    
    # 6.校验总和
    sum1 = 0;
    for v in result:
        sum1 += v[1]
    print("info: sum=%s" % sum1)
        
    print("*** OVER ***")

